import json
import os
import requests
from bs4 import BeautifulSoup

from utils import Download

with open('GME_Variome.json', 'r')as f:
    data_source = json.load(f)
baseurl = data_source['baseurl']

response = requests.get(baseurl)
if response.status_code != 200:
    raise ValueError

soup = BeautifulSoup(response.text, 'html.parser')

tag_a_list = soup.find(class_='container').find_all('a')

data_source['files'] = []
for index, tag_a in enumerate(tag_a_list):
    url = 'http://igm.ucsd.edu/gme/' + tag_a['href']
    filename = url.split('/')[-1]
    path = os.path.join('GME_Variome', filename)

    download = Download(url=url, dirName='GME_Variome')
    md5 = download.start()

    file = {
        'filename': filename,
        'path': path,
        'md5': md5
    }
    data_source['files'].insert(index, file)

else:
    with open('GME_Variome.json', 'w')as f:
        json.dump(data_source, f)
